#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Author  : lkm

"""
    网络爬虫：采集stockx上的运动鞋销售数据
"""

import json
import requests
import pymysql
import configparser

url = r"https://stockx.com/api/browse?page={}&sort=deadstock_sold&productCategory=sneakers&order=DESC"
engine = None

def configure_info():
    _file = r'configure.ini'
    _section = r'mysql'
    _item = ['HOST', 'PORT', 'USER', 'PASSWD', 'DB']

    cf = configparser.ConfigParser()
    cf.read(_file)
    host = cf.get(_section, 'host')
    port = int(cf.get(_section, 'port'))
    user = cf.get(_section, 'user')
    passwd = cf.get(_section, 'passwd')
    db = cf.get(_section, 'db')

    global engine
    engine = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, charset="utf8")

def craw_sneakers():
    configure_info()
    header = {
        'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",
        "content-type": "application/json",
        "cookie": "__cfduid=d087c81fb9bddd6a00ee009d8b0205f8d1592754766; "
                  "_pxhd=d50dc446b073b929ce9e60da5c41c8ddfcac23e5deb5bad3b14a390d152d6ffa:"
                  "404bc731-b3d7-11ea-844e-e16f13438b99; stockx_market_country=CN; _ga=GA1.2.2003174848."
                  "1592754794; _gid=GA1.2.673007109.1592754794; _rdt_uuid=1592754794186.b84a98c4-0721-4fa0-"
                  "b5d3-a1f2473e8ff5; _ALGOLIA=anonymous-7127a46b-bd0b-464f-ac59-3695aac876c7; is_gdpr=false; "
                  "cookie_policy_accepted=true; stockx_ip_region=CN; tracker_device=77992bea-ab4b-4432-"
                  "8920-a50bf2141b61; show_all_as_number=false; brand_tiles_version=v1; show_bid_education=v2;"
                  " show_bid_education_times=1; multi_edit_option=beatLowestAskBy; show_watch_modal=false; "
                  "intl_payments=true; bulk_shipping_enabled=false; below_retail_type=; bid_ask_button_type=; "
                  "show_chatbot=false; related_products_length=v2; riskified_recover_updated_verbiage=true; "
                  "show_how_it_works=true; sell_ui_refresh=undefined; browse_page_tile_size_update_web=false; "
                  "product_page_affirm_callout_enabled_web=false; _gcl_au=1.1.565611884.1592754797; "
                  "ajs_anonymous_id=%223ca2f698-8a33-4236-b4fb-fc0a9cbf014e%22; IR_gbd=stockx.com; rskxRunCookie=0; "
                  "rCookie=si9ix6hjd4bqxut1lrfit8kbp93ggn; language_code=zh; stockx_selected_locale=zh; "
                  "stockx_selected_region=CN; stockx_dismiss_modal=true; stockx_dismiss_modal_set=2020-06-21T15%3A53%3A41."
                  "580Z; stockx_dismiss_modal_expiration=2021-06-21T15%3A53%3A41.579Z; _sp_ses.1a3e=*; _dd_r=0; "
                  "stockx_session=4172d251-812a-443e-bd3c-d1014ffe03c7; _pk_ses.421.1a3e=*; stockx_homepage=sneakers; "
                  "lastRskxRun=1592832066789; _gat=1; _sp_id.1a3e=299e413e-cf78-4300-89f5-ca34210c1b2b.1592754794.2."
                  "1592832130.1592755582.0df98d4d-2034-4a3c-800e-a43fd992cea4; IR_9060=1592832131732%7C0%7C1592831826"
                  "710%7C%7C; IR_PI=5334a101-b3d7-11ea-a287-061915aeeca6%7C1592918531732; _pk_id.421.1a3e=472978623ec"
                  "dc323.1592754799.2.1592832133.1592831827."
    }
    cursor = engine.cursor()
    for i in range(1, 26):
        URL = url.format(i)
        response = requests.get(URL, headers=header)
        text = json.loads(response.text)
        keys = ['id', 'uuid', 'brand', 'category', 'colorway', 'condition', 'dataType',
                'minimumBid', 'gender', 'name', 'productCategory', 'releaseDate', 'releaseTime',
                'retailPrice', 'shoe', 'shortDescription', 'styleId', 'tickerSymbol', 'title',
                'type', 'urlKey', 'year']
        for j in text["Products"]:
            test = []
            for k in keys:
                test.append(j[k])
            sql = r"insert into sneakers values{}".format(tuple(test))
            print(sql)
            try:
                cursor.execute(sql)
                engine.commit()
            except Exception as e:
                print(e)
                engine.rollback()
    cursor.close()

if __name__ == "__main__":
    craw_sneakers()

